# -*- coding: utf-8 -*-
# =============================================================================
#     FileName: redis_expire_helper.py
#         Desc: 用于清理Redis服务器上指定前缀的KEY，通过循环SCAN方式降低清理过程对服务器的影响。
#       Author: GGA
#        Email:
#     HomePage:
#      Version: 1.0.1
#   LastChange: 2020-07-31
#      History:
# =============================================================================
import time
import redis
import random
from collections import namedtuple
import abc
from utils.logger_helper import LoggerHelper

app_name = "redis_expire_helper"
app_version = "1.0"
logger = LoggerHelper.get_logger()

RedisConfig = namedtuple("RedisConfig", field_names=["redis_host", "redis_port", "redis_password", "redis_db"], )


class BaseCleaner(metaclass=abc.ABCMeta):
    def __init__(self, redis_host, redis_port,
                 redis_password, redis_db,
                 key_pattern, expire_seconds, random_seconds,
                 scan_size=1000, scan_interval=0.1,
                 pipe_size=100, log_sample=0.01):
        self.redis_host = redis_host
        self.redis_port = redis_port
        self.redis_password = redis_password
        self.redis_db = redis_db
        self.key_pattern = key_pattern
        self.expire_seconds = expire_seconds
        self.random_seconds = random_seconds
        self.scan_size = scan_size
        self.scan_interval = scan_interval
        self.pipe_size = pipe_size
        self.key_log_sample = log_sample
        self.redis_pool = redis.ConnectionPool(
            host=redis_host,
            port=redis_port,
            db=redis_db,
            password=redis_password,
            max_connections=2
        )

    def _get_redis_connection(self):
        """
        从Redis Pool中获取连接，避免重复建立Redis连接
        :return:
        """
        return redis.Redis(connection_pool=self.redis_pool, decode_responses=True)

    def _check_connection(self):
        redis_conn = self._get_redis_connection()
        try:
            redis_conn.ping()
            return True
        except Exception as _:
            logger.info("无法正常连接Redis，请检查Redis配置信息是否正确。")
            exit(1)

    def _scan_keys(self, start_post=0):
        """
        范围扫描匹配的键值, 并返回匹配的KEYS

        :param start_post:  扫描开始位置点
        :return: stop_post,match_keys
            stop_post: 本次扫描的结束位置
            match_keys：满足条件的KEYS
        """
        redis_conn = self._get_redis_connection()
        stop_post, match_keys = redis_conn.scan(start_post, self.key_pattern, self.scan_size)
        logger.info("在{}:{}实例上扫描起始点:{},扫描结束点:{},扫描数量:{},匹配Key数量:{}".format(
            self.redis_host, self.redis_port,
            start_post, stop_post, self.scan_size, len(match_keys)))
        return stop_post, match_keys

    def _get_idle_seconds(self, key_names):
        key_items = dict()
        if len(key_names) == 0:
            return key_items
        redis_conn = self._get_redis_connection()
        redis_pipe = redis_conn.pipeline()
        for key_name in key_names:
            redis_pipe.object("IDLETIME", key=key_name)
        pipe_result = redis_pipe.execute()
        for key_index, key_name in enumerate(key_names):
            key_items[key_name] = pipe_result[key_index]
        return key_items

    def _get_expire_seconds(self, key_names):
        key_items = dict()
        if len(key_names) == 0:
            return key_items
        redis_conn = self._get_redis_connection()
        redis_pipe = redis_conn.pipeline()
        for key_name in key_names:
            redis_pipe.ttl(key_name)
        pipe_result = redis_pipe.execute()
        for key_index, key_name in enumerate(key_names):
            key_items[key_name] = pipe_result[key_index]
        return key_items

    def _get_long_idle_keys(self, key_names, long_idle_seconds):
        """
        通过PIPLINE来批量获取指定KEYS的空闲时间，并返回空闲时间超过阈值的KEYS
        :param key_names:
        :return:
        """
        match_keys = list()
        key_items = self._get_idle_seconds(key_names=key_names)
        for key_name, idle_seconds in key_items.items():
            if idle_seconds is not None and int(idle_seconds) > long_idle_seconds:
                if random.random() < self.key_log_sample:
                    logger.info("键值{}当前的空闲时间为{},超过阈值{}".format(
                        key_name, idle_seconds, long_idle_seconds))
                match_keys.append(key_name)
        return match_keys

    def _get_unexpired_keys(self, key_names):
        """
        通过PIPLINE来批量获取指定KEYS的空闲时间，并返回未设置过期时间的KEY.
        :param key_names:
        :return:
        """
        match_keys = list()
        key_items = self._get_expire_seconds(key_names=key_names)
        for key_name, expire_seconds in key_items.items():
            if expire_seconds is not None and expire_seconds == -1:
                if random.random() < self.key_log_sample:
                    logger.info("键值{}未设置过期时间".format(key_name))
                match_keys.append(key_name)
            pass
        return match_keys

    def _get_long_expired_keys(self, key_names, long_expire_seconds):
        """
        通过PIPLINE来批量获取指定KEYS的过期时间，并返回过期时间超过阈值的KEYS
        :param key_names:
        :param long_expire_seconds
        :return:
        """
        match_keys = list()
        key_items = self._get_expire_seconds(key_names=key_names)
        for key_name, expire_seconds in key_items.items():
            if expire_seconds is not None and expire_seconds > long_expire_seconds:
                if random.random() < self.key_log_sample:
                    logger.info("键值{}的过期时间为{},超过阈值{}".format(
                        key_name, expire_seconds, long_expire_seconds
                    ))
                match_keys.append(key_name)
            pass
        return match_keys

    def _spilt_keys_to_pipe_groups(self, key_names):
        """
        将KEY按照pipe操作批次拆分多个pipe组
        :param key_names:
        :return:
        """
        pipe_groups = []
        group_keys = []
        for key_name in key_names:
            group_keys.append(key_name)
            if len(group_keys) == self.pipe_size:
                pipe_groups.append(group_keys)
                group_keys = []
            pass
        if len(group_keys) > 0:
            pipe_groups.append(group_keys)
        return pipe_groups

    @abc.abstractmethod
    def _get_need_expired_keys(self, key_names):
        pass

    @abc.abstractmethod
    def _check_config(self):
        pass

    def _expire_keys(self, key_names):
        """
        通过PIPLINE来批量设置指定KEYS的过期时间
        :param key_names:
        :return:
        """
        if len(key_names) == 0:
            return
        redis_conn = self._get_redis_connection()
        redis_pipe = redis_conn.pipeline()
        for key_name in key_names:
            expire_seconds = self.expire_seconds + random.randint(0, self.random_seconds)
            redis_pipe.expire(key_name, expire_seconds)
            if random.random() < self.key_log_sample:
                logger.info("设置key {} 在{}秒后过期.".format(key_name, expire_seconds))
        redis_pipe.execute()

    def _loop_expire_keys(self):
        logger.info("开始扫描{}实例".format(self.redis_host, self.redis_port, ))
        start_post = 0
        stop_post = 1
        while stop_post > 0:
            stop_post, match_keys = self._scan_keys(start_post=start_post)
            batch_groups = self._spilt_keys_to_pipe_groups(key_names=match_keys)
            for batch_keys in batch_groups:
                dt1 = time.time()
                need_expire_keys = self._get_need_expired_keys(key_names=batch_keys)
                dt2 = time.time()
                self._expire_keys(key_names=need_expire_keys)
                dt3 = time.time()
                logger.info(
                    "在{}:{}实例上,检查{}个KEY操作耗时: {} ms,设置{}个KEY操作耗时: {} ms".format(
                        self.redis_host, self.redis_port,
                        len(batch_keys), (dt2 - dt1) * 1000, len(need_expire_keys), (dt3 - dt2) * 1000,
                    ))
            start_post = stop_post
            time.sleep(self.scan_interval)
        logger.info("完成扫描{}:{}实例".format(self.redis_host, self.redis_port, ))

    def expire_keys(self):
        self._check_config()
        self._check_connection()
        self._loop_expire_keys()


class ExpiredKeyCleaner(BaseCleaner):
    def __init__(self, redis_host="127.0.0.1", redis_port=6379,
                 redis_password="", redis_db=0,
                 scan_size=1000, scan_interval=0.1,
                 pipe_size=100, log_sample=0.01):
        super().__init__(
            redis_host=redis_host, redis_port=redis_port,
            redis_password=redis_password, redis_db=redis_db,
            key_pattern="*", expire_seconds=99999999999, random_seconds=0,
            scan_size=scan_size, scan_interval=scan_interval,
            pipe_size=pipe_size, log_sample=log_sample,
        )
        pass

    def _check_config(self):
        pass

    def _get_need_expired_keys(self, key_names):
        return []


class LongIdleKeyCleaner(BaseCleaner):
    def __init__(self, key_pattern, long_idle_seconds: int,
                 expire_seconds: int, random_seconds: int,
                 redis_host="127.0.0.1", redis_port=6379,
                 redis_password="", redis_db=0,
                 scan_size=1000, scan_interval=0.1,
                 pipe_size=100, log_sample=0.01):
        super().__init__(
            redis_host=redis_host, redis_port=redis_port,
            redis_password=redis_password, redis_db=redis_db,
            key_pattern=key_pattern, expire_seconds=expire_seconds,
            random_seconds=random_seconds,
            scan_size=scan_size, scan_interval=scan_interval,
            pipe_size=pipe_size, log_sample=log_sample
        )
        self.long_idle_seconds = long_idle_seconds

    def _check_config(self):
        pass

    def _get_need_expired_keys(self, key_names):
        self._get_long_idle_keys(key_names, long_idle_seconds=self.long_idle_seconds)


class UnexpiredKeyExpire(BaseCleaner):
    def __init__(self, key_pattern,
                 expire_seconds: int, random_seconds: int,
                 redis_host="127.0.0.1", redis_port=6379,
                 redis_password="", redis_db=0,
                 scan_size=1000, scan_interval=0.1,
                 pipe_size=100, log_sample=0.01):
        super().__init__(
            redis_host=redis_host, redis_port=redis_port,
            redis_password=redis_password, redis_db=redis_db,
            key_pattern=key_pattern, expire_seconds=expire_seconds,
            random_seconds=random_seconds,
            scan_size=scan_size, scan_interval=scan_interval,
            pipe_size=pipe_size, log_sample=log_sample,
        )
        pass

    def _check_config(self):
        pass

    def _get_need_expired_keys(self, key_names):
        return self._get_unexpired_keys(key_names=key_names)


class LongExpiredKeyExpire(BaseCleaner):
    def __init__(self, key_pattern, long_expire_seconds,
                 expire_seconds: int, random_seconds: int,
                 redis_host="127.0.0.1", redis_port=6379,
                 redis_password="", redis_db=0,
                 scan_size=1000, scan_interval=0.1,
                 pipe_size=100, log_sample=0.01):
        super().__init__(
            redis_host=redis_host, redis_port=redis_port,
            redis_password=redis_password, redis_db=redis_db,
            key_pattern=key_pattern, expire_seconds=expire_seconds,
            random_seconds=random_seconds,
            scan_size=scan_size, scan_interval=scan_interval,
            pipe_size=pipe_size, log_sample=log_sample
        )
        self.long_expire_seconds = long_expire_seconds

    def _check_config(self):
        pass

    def _get_need_expired_keys(self, key_names):
        return self._get_long_expired_keys(key_names=key_names, long_expire_seconds=self.long_expire_seconds)
